package com.spider.common.task.service;

import com.eclipsesource.v8.V8;
import com.spider.common.bean.enums.TenderUrlEnum;
import com.spider.common.bean.tender.TenderSearchVo;
import com.spider.common.bean.tender.TenderVo;
import com.spider.common.utils.StringUtils;
import com.xuxueli.crawler.XxlCrawler;
import com.xuxueli.crawler.parser.PageParser;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.DateUtils;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;

import javax.script.ScriptEngine;
import javax.script.ScriptEngineManager;
import javax.script.ScriptException;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URLEncoder;
import java.util.*;

/**
 * @author marver
 * @Description 招标网站采集
 * @date 2019/4/8 15:11
 */
@Service
public class TenderTask {
    private static Logger logger = LoggerFactory.getLogger(TenderTask.class);
    private static final String url = "http://www.chinabidding.cc";
    private static List<Map<String, Object>> mapList;


    /**
     * 采集招标采购导航网
     * @param searchVo
     */
    private static void okCisInfo(TenderSearchVo searchVo){
        Map paramMap = new HashMap();
        paramMap.put("result-keyword-or-form", searchVo.getSearch());
        paramMap.put("result-class-type-form", "brn");
        paramMap.put("result-content-type-form", "1");
        paramMap.put("result-auto-type-form", "1");
        paramMap.put("result-time-type-form", "4");
        paramMap.put("search-start-time-input", "2019-01-08");
        paramMap.put("search-end-time-input", DateUtils.formatDate(new Date()));
        paramMap.put("showType", "1");
        paramMap.put("countnum", "67802");
        paramMap.put("page", searchVo.getPage());
        paramMap.put("size", searchVo.getPageSize());

        XxlCrawler crawler = new XxlCrawler.Builder()
                .setUrls(TenderUrlEnum.OKCIS.getUrl())
                .setWhiteUrlRegexs(TenderUrlEnum.OKCIS.getUrl())
                .setParamMap(paramMap)
                .setIfPost(true)
                .setThreadCount(3)
                .setAllowSpread(false) //不允许扩散爬取
                .setPageParser(new PageParser<TenderVo>() {
                    @Override
                    public void parse(Document html, Element pageVoElement, TenderVo pageVo) {
                        // 解析封装 PageVo 对象
                        parseInfo(pageVo);
                    }
                }).build();
        crawler.start(true);
    }

    /**
     * 解析结果
     * @param tenderVo
     */
    private static void parseInfo(TenderVo tenderVo){
        mapList = new ArrayList<>();
        for (int i = 0; i < tenderVo.getTitle().size(); i++) {
            Map<String, Object> map = new HashMap();
            map.put("title", tenderVo.getTitle().get(i));
            map.put("titleUrl", tenderVo.getTitleUrl().get(i));
            mapList.add(map);
        }
    }

    /**
     * 采集中国采购招标网信息
     * @param searchVo
     */
    private static List<Map<String, Object>> chinabiddInfo(TenderSearchVo searchVo){
        List<Map<String, Object>> list = new ArrayList<>();
        try {
            CloseableHttpClient client = HttpClients.createDefault();
            HttpGet get = new HttpGet(url);
            //模拟浏览器
            get.setHeader("Accept", "Accept text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
            get.setHeader("Accept-Charset", "GB2312,utf-8;q=0.7,*;q=0.7");
            get.setHeader("Accept-Encoding", "gzip, deflate");
            get.setHeader("Accept-Language", "zh-cn,zh;q=0.5");
            get.setHeader("Connection", "keep-alive");
            get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:6.0.2) Gecko/20100101 Firefox/6.0.2");
            CloseableHttpResponse response = client.execute(get);
            //拿到第一次请求返回的JS
            if(response.getStatusLine().getStatusCode()==521){
                HttpEntity entity = response.getEntity();
                String html=EntityUtils.toString(entity,"utf-8");
                logger.debug(html);
                //处理从服务器返回的JS，并执行
                String js=html.trim().replace("<script>", "").replace("</script>", "").replace("eval(y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]||(\"_\"+y)})","y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]||(\"_\"+y)}");
                V8 runtime = V8.createV8Runtime();
                String result=runtime.executeStringScript(js);
                logger.debug(result);
                //第二次处理JS并执行
                String jsl_clearance = getCookie(result, runtime);
                URI uri = new URI(chinabiddingInfo(searchVo));
                get.setURI(uri);
                get.setHeader("Cookie",jsl_clearance);

                response = client.execute(get);
            }
            //拿到最终想要的页面
            HttpEntity entity = response.getEntity();
            String res = EntityUtils.toString(entity,"utf-8");
            Document doc = Jsoup.parse(res);
            Elements elements = doc.select("#center_box > div.lists_box > div.left_box > div.lists_center > ul > li > a");
            for (int i = 0; i < elements.size() ; i++) {
                Map<String, Object> map = new HashMap<>();
                map.put("title", elements.eachText().get(i));
                map.put("titleUrl", elements.eachAttr("href").get(i));
                list.add(map);
            }
        } catch (Exception e) {
            logger.error("采集失败"+e.getMessage());
        }
        return list;
    }

    /**
     * 获取cookie
     * @param result
     * @param runtime
     * @return
     */
    private static String getCookie(String result, V8 runtime) {
        String jsl_pre = result.substring(result.indexOf("__jsl_clearance=") + 16,
                result.indexOf("|'+(function(){var") + 1);
        String bac = (result.substring(result.indexOf("|'+(function(){"), result.indexOf("+';Expires=") - 23)
                + result.substring(result.indexOf("+';Expires=") - 16, result.indexOf("+';Expires=") - 4))
                .replace("|'+(function(){", "").replaceAll("window", "'Chrome'");
        String jsl_bac = runtime.executeStringScript(bac);
        String jsl_clearance = "__jsl_clearance=" +jsl_pre + jsl_bac+";";
        logger.debug(jsl_clearance);
        return jsl_clearance;
    }

    private static String chinabiddingInfo(TenderSearchVo searchVo){
        Map paramMap = new HashMap();
        paramMap.put("keyword", searchVo.getSearch());
        paramMap.put("h_lx", "0");
        paramMap.put("h_province", "0");
        paramMap.put("date", "30");
        paramMap.put("search_field", "0");
        paramMap.put("sign", "deef9e3faa89443d8b0b590fa320f941");
        paramMap.put("page", searchVo.getPage());
        String geturl = TenderUrlEnum.CHINABIDDING.getUrl();
        try {
            geturl += "?";
            String key;
            for(Iterator var3 = paramMap.keySet().iterator(); var3.hasNext(); geturl = geturl + key + "=" + URLEncoder.encode(paramMap.get(key).toString(), "utf-8") + "&") {
                key = (String)var3.next();
            }
            geturl = geturl.substring(0, geturl.length() - 1);
        } catch (UnsupportedEncodingException e) {
            e.printStackTrace();
        }
        return geturl;
    }

    /**
     * 采集招标总任务
     * @param searchVo
     * @return
     */
    public List<Map<String, Object>> getTenderData(TenderSearchVo searchVo){
        List<Map<String, Object>> mapArrayList = new ArrayList<>();
        String[] searchs = {"档案", "光盘库", "光存储", "数据备份"};

        if(StringUtils.isEmpty(searchVo.getSearch())){
            for(String search : searchs){
                searchVo.setSearch(search);
                okCisInfo(searchVo);
                mapArrayList.addAll(mapList);
                List<Map<String, Object>> list = chinabiddInfoTwo(searchVo);
                mapArrayList.addAll(list);
            }
        }else{
            okCisInfo(searchVo);
            mapArrayList.addAll(mapList);
            List<Map<String, Object>> list = chinabiddInfoTwo(searchVo);
            mapArrayList.addAll(list);
        }
        return mapArrayList;
    }

    /**
     * 第二种方法解析 中国采购招标网信息
     * @param searchVo
     */
    public static List<Map<String, Object>> chinabiddInfoTwo(TenderSearchVo searchVo) {
        List<Map<String, Object>> list = new ArrayList<>();
        try {
            Thread.sleep(2000);
            CloseableHttpClient client = HttpClients.createDefault();
            HttpGet get = new HttpGet(url);
            //模拟浏览器
            get.setHeader("Accept", "Accept text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
            get.setHeader("Accept-Charset", "GB2312,utf-8;q=0.7,*;q=0.7");
            get.setHeader("Accept-Encoding", "gzip, deflate");
            get.setHeader("Accept-Language", "zh-cn,zh;q=0.5");
            get.setHeader("Connection", "keep-alive");
            get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:6.0.2) Gecko/20100101 Firefox/6.0.2");
            CloseableHttpResponse response = client.execute(get);
            //拿到第一次请求返回的JS
            if(response.getStatusLine().getStatusCode()==521){
                HttpEntity entity = response.getEntity();
                String resHtml = EntityUtils.toString(entity);

                //第二次处理JS并执行
                String jsl_clearance = getJslClearance(resHtml);;
                get.setHeader("Cookie",jsl_clearance);
                URI uri = new URI(chinabiddingInfo(searchVo));
                get.setURI(uri);
                response = client.execute(get);
            }
            //拿到最终想要的页面
            HttpEntity entity = response.getEntity();
            String res = EntityUtils.toString(entity,"utf-8");
            Document doc = Jsoup.parse(res);
            Elements elements = doc.select("#center_box > div.lists_box > div.left_box > div.lists_center > ul > li > a");
            for (int i = 0; i < elements.size() ; i++) {
                Map<String, Object> map = new HashMap<>();
                map.put("title", elements.eachText().get(i));
                map.put("titleUrl", elements.eachAttr("href").get(i));
                list.add(map);
            }
        } catch (Exception e) {
            logger.error("采集失败"+e.getMessage());
        }
        return list;
    }

    private static String getJslClearance(String body) {
        String jsl_clearance = "";
        ScriptEngineManager manager = new ScriptEngineManager();
        //得到脚本引擎
        ScriptEngine engine = manager.getEngineByName("JavaScript");
        //处理加密js
        String js = body.trim().replace("<script>", "")
                .replace("</script>", "")
                .replace("eval(y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]||(\"_\"+y)}))",
                        "y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]||(\"_\"+y)})");
        try {
            //得到解密后的js
            String result = (String) engine.eval(js);

            String jsl_pre = result.substring(result.indexOf("__jsl_clearance=") + 16,
                    result.indexOf("|'+(function(){var") + 1);
            String bac = (result.substring(result.indexOf("|'+(function(){"), result.indexOf("+';Expires=") - 23)
                    + result.substring(result.indexOf("+';Expires=") - 16, result.indexOf("+';Expires=") - 4))
                    .replace("|'+(function(){", "").replaceAll("window", "'Chrome'");
            // 新增解决document问题
            if(bac.indexOf(".firstChild.href;var ")>0){
                String varTem = bac
                        .substring(bac.indexOf(".firstChild.href;var "), bac.indexOf(".match(/https?:\\/\\//)[0]"))
                        .replace(".firstChild.href;var ", "").split("=")[0];
                String varDocument = bac
                        .substring(bac.indexOf(".firstChild.href;var "), bac.indexOf(".match(/https?:\\/\\//)[0]"))
                        .replace(".firstChild.href;var ", "").split("=")[1];
                String varA = bac.substring(bac.indexOf("<a href=\\'/\\'>"), bac.indexOf("</a>';"))
                        .replace("<a href=\\'/\\'>", "");
                //取任意一次作为模板
                String documentModel = "document.createElement('div');_15.innerHTML='<a href=\\'/\\'>_t</a>';_15=_15.firstChild.href;var _1L=_15.match(/https?:\\/\\//)[0];_15=_15.substr(_1L.length).toLowerCase()";
                documentModel = documentModel.replace("_15", varDocument).replace("_1L", varTem).replace("_t", varA);

                bac = bac.replace(documentModel, "'www.chinabidding.cc/'");
            }

            String jsl_bac = (String) engine.eval(bac);
            jsl_clearance = "__jsl_clearance=" +jsl_pre + jsl_bac+";";
            logger.debug(jsl_clearance);
        } catch (ScriptException e) {
            e.printStackTrace();
        }
        return jsl_clearance;
    }

    public static void main(String[] args) {
        TenderTask tenderTask = new TenderTask();
        tenderTask.okCisInfo(new TenderSearchVo());
        System.out.println(tenderTask.mapList);
    }
}
